from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_extraction.text import CountVectorizer
import jieba
from sklearn.feature_extraction.text import TfidfVectorizer


def datasets_demo():
    """
    sklearn数据集使用
    :return:
    """
    # 获取数据集
    iris = load_iris()
    print("鸢尾花数据集：\n", iris)
    print("查看数据集描述：\n", iris["DESCR"])
    print("查看特征值的名字：\n", iris.feature_names)

    # 数据集划分
    x_train, x_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2)
    print("查看训练特征值：\n", x_train, x_train.shape)

    # fetch_20newsgroups(data_home='C:\\Users\\Administrator\\sklearn_data', subset='train')

    return None


def dict_demo():
    """
    字典特征处理
    :return:
    """
    data = [{'city': '北京', 'temperature': 100}, {'city': '上海', 'temperature': 60},
            {'city': '深圳', 'temperature': 30}]
    # 1、实例化一个转换器类
    transfer = DictVectorizer()
    data_new = transfer.fit_transform(data)
    print("特征：\n", data_new.toarray(), type(data_new))
    print("特征名称：\n", transfer.feature_names_)
    return None


def count_demo():
    """
    文本特征抽取
    :return:
    """
    data = ["life is short,i like like python", "life is too long,i dislike python"]
    transfer = CountVectorizer()
    data_new = transfer.fit_transform(data)
    print("data_new:\n", data_new.toarray())
    print("特征名称:\n", transfer.get_feature_names_out())
    return None


def count_chinese_demo():
    """
    中文文本特征抽取
    :return:
    """
    data = ["我 爱 北京 天安门", "天安门 上 太阳 升"]
    transfer = CountVectorizer()
    data_new = transfer.fit_transform(data)
    print("data_new:\n", data_new.toarray())
    print("特征名称:\n", transfer.get_feature_names_out())
    return None


def count_chinese_demo2():
    """
    中文文本特征抽取
    :return:
    """
    data = ["一种还是一种今天很残酷，明天更残酷，后天很美好，但绝对大部分是死在明天晚上，所以每个人不要放弃今天。",
            "我们看到的从很远星系来的光是在几百万年之前发出的，这样当我们看到宇宙时，我们是在看它的过去。",
            "如果只用一种方式了解某样事物，你就不会真正了解它。了解事物真正含义的秘密取决于如何将其与我们所了解的事物相联系。"]
    data_jieba = []
    for datum in data:
        data_jieba.append(cut_sentence(datum))

    transfer = CountVectorizer(stop_words=["因为", "所以"])
    data_new = transfer.fit_transform(data_jieba)
    print("data_new:\n", data_new.toarray())
    print("特征名称:\n", transfer.get_feature_names_out())
    return None


def cut_sentence(text):
    """
    中文分词
    :param text:
    :return:
    """
    return " ".join(list(jieba.cut(text)))


def tfidf_demo():
    """
    tfidf特征提取
    :return:
    """
    data = ["一种还是一种今天很残酷，明天更残酷，后天很美好，但绝对大部分是死在明天晚上，所以每个人不要放弃今天。",
            "我们看到的从很远星系来的光是在几百万年之前发出的，这样当我们看到宇宙时，我们是在看它的过去。",
            "如果只用一种方式了解某样事物，你就不会真正了解它。了解事物真正含义的秘密取决于如何将其与我们所了解的事物相联系。"]
    data_jieba = []
    for datum in data:
        data_jieba.append(cut_sentence(datum))

    transfer = TfidfVectorizer()
    data_new = transfer.fit_transform(data_jieba)
    print("data_new:\n", data_new.toarray())
    print("特征名称:\n", transfer.get_feature_names_out())
    return None


if __name__ == '__main__':
    # 代码1：sklearn数据集使用
    # datasets_demo()
    # dict_demo()
    # count_demo()
    # count_chinese_demo()
    # count_chinese_demo2()
    tfidf_demo()